"""
@Filename       : meme_preprocess.py
@Create Time    : 2021/9/23 9:40
@Author         : Rylynn
@Description    : 

"""
import time
import random


def load_memetracker(filepath):
    meme_dict = {}
    url_count_dict = {}
    content_dict = {}
    url_id = {}
    URL_MAX_ID = 1
    with open(filepath, encoding='utf8', mode='r') as f:
        for _ in range(5):
            f.readline()

        line = f.readline()
        while line:
            line = f.readline()
            cls, _, content, clusterId = line.strip().split('\t')
            # meme_dict[clusterId] = []
            # content_dict[clusterId] = content
            flag = True
            for _ in range(eval(cls)):
                line = f.readline()
                _, url_number, _, meme_id = line.strip().split('\t')
                meme_dict[meme_id] = []
                exist_user = set()

                for _ in range(eval(url_number)):
                    line = f.readline()
                    t, mention_number, type, url = line.strip().split('\t')
                    url = url.split('/')[2]
                    if not url_count_dict.get(url):
                        url_count_dict[url] = 1
                        url_id[url] = URL_MAX_ID
                        URL_MAX_ID += 1
                    else:
                        url_count_dict[url] += 1

                    if not url_id[url] in exist_user:
                        if flag:
                            meme_dict[meme_id].append((url_id[url], int(time.mktime(time.strptime(t, '%Y-%m-%d %H:%M:%S')))))
                            exist_user.add(url_id[url])

                flag = False
                line = f.readline()

    new_meme_dict = {}

    active_users = list(filter(lambda x: x[1] >= 60, url_count_dict.items()))
    active_users = set(map(lambda x: url_id[x[0]], active_users))

    for meme, users in meme_dict.items():
        new_users = []
        for user, t in users:
            if user in active_users:
                new_users.append((user, t))
        if 5 <= len(new_users) <= 500:
            new_meme_dict[meme] = new_users

    final_user = set()

    for meme, users in new_meme_dict.items():
        for user, _ in users:
            final_user.add(user)
    #
    print(len(final_user))
    print(len(new_meme_dict))
    print(sum(map(lambda x: len(x), new_meme_dict.values())) / len(new_meme_dict))

    # with open('../../../data/memetracker/content.txt', 'w') as f:
    #     for cluster_id, content in content_dict.items():
    #         f.write("{}\t{}\n".format(cluster_id, content))
    #
    # final_cascade = list(new_meme_dict.items())
    # random.shuffle(final_cascade)
    #
    # train_cascade = final_cascade[:int(len(final_cascade) * 0.8)]
    # test_cascade = final_cascade[int(len(final_cascade) * 0.8): int(len(final_cascade) * 0.9)]
    # valid_cascade = final_cascade[int(len(final_cascade) * 0.9):]

    # def write_cascade(cascade, filename):
    #     with open(filename, 'w') as f:
    #         for k, c in cascade:
    #             f.write('{} '.format(k))
    #             for u, t in c:
    #                 f.write('{},{} '.format(u, t))
    #             f.write('\n')
    #
    # write_cascade(train_cascade, '../../../data/memetracker/cascade.txt')
    # write_cascade(test_cascade, '../../../data/memetracker/cascadetest.txt')
    # write_cascade(valid_cascade, '../../../data/memetracker/cascadevalid.txt')

def main():
    meme_path = "../../../data/raw/clust-qt08080902w3mfq5.txt"
    load_memetracker(meme_path)


if __name__ == '__main__':
    main()